{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "196a4420-a3b4-49b2-996d-ca4200e0ea4f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "386948b3-65a1-428f-bbfd-197fb0abb78e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def show(img, figsize=(2,2)):\n",
    "    x, y = figsize # tuple 解包避免: argument must be a string, a bytes-like object or a real number, not 'tuple'\n",
    "    plt.figure(figsize=(x, y))\n",
    "    plt.imshow(img)\n",
    "    plt.axis(\"off\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8849786-7ce4-426d-bcc6-8cfd15c147ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision.io import read_image\n",
    "from torchvision.transforms import ToPILImage\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a472ee4-16f4-426b-b3cc-391f3dad10a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# torchvision.transforms 提供了多个通用转换, 如 ToTensor, Lambda, ToPILImage\n",
    "# 将标签数字转为one-hot的张量, 便于矩阵运算\n",
    "\n",
    "lable = 5\n",
    "l_tensor = torch.zeros(10, dtype=torch.float).scatter_(dim=0, index=torch.tensor(lable), value=1)\n",
    "print(f'l_tensor：{l_tensor}')\n",
    "\n",
    "root = f\"{'/'.join(os.getcwd().split('/')[:3])}/hdfs/training-data\"\n",
    "i_tensor = read_image(f'{root}/mytd/cifar-train-2.jpg')\n",
    "print(f'i_tensor: {i_tensor.shape}')\n",
    "\n",
    "img = ToPILImage()(i_tensor)\n",
    "show(img, figsize=(0.4, 0.4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "379bcacd-7630-46df-b7a8-1d29b34f7a80",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets, transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c8dc8c5-c843-4cd9-88db-732747101936",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 构建神经网络\n",
    "\n",
    "device = torch.accelerator.current_accelerator().type if torch.accelerator.is_available() else \"cpu\"\n",
    "print(f\"Using {device} device\")\n",
    "\n",
    "class NeuralNetwork(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.flatten = nn.Flatten()\n",
    "        self.linear_relu_stack = nn.Sequential(\n",
    "            nn.Linear(28*28, 512),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(512, 512),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(512, 10),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.flatten(x)\n",
    "        logits = self.linear_relu_stack(x)\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd2077a0-b044-4bdd-85db-c716507a446c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 使用模型\n",
    "model = NeuralNetwork().to(device)\n",
    "print(model)\n",
    "X = torch.rand(1, 28, 28, device=device)  # 随机一张28*28的灰度图\n",
    "logits = model(X) # 模型接收数据并向前计算, 得到shape(1, 10)的张量\n",
    "pred_probab = nn.Softmax(dim=1)(logits)  # 在dim=1维度上使用softmax将模型输出结果重新分配, 使每个值取[0, 1]内且sum=1, 即将数值转百分比\n",
    "y_pred = pred_probab.argmax(1) # 从张量中dim=1维取最大值所在的索引\n",
    "\n",
    "torch.set_printoptions(sci_mode=False)\n",
    "print(f\"logits: {logits}\")\n",
    "print(f\"pred_probab: {pred_probab}\")\n",
    "print(f\"Predicted class: {y_pred}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d619fbb0-ccc1-43fc-94aa-aadc4a877421",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 模型步骤拆分\n",
    "\n",
    "input_image = torch.rand(3,28,28)  # 3个数据样本\n",
    "print(input_image.size())\n",
    "\n",
    "flatten = nn.Flatten()  # 扁平化, 即dim>=1的数据扁平到dim=1\n",
    "flat_image = flatten(input_image)\n",
    "print(flat_image.size())\n",
    "\n",
    "layer1 = nn.Linear(in_features=28*28, out_features=20)  # 线性层处理 (n, x) * (x, m) = (n, m)\n",
    "hidden1 = layer1(flat_image)\n",
    "print(hidden1.size())\n",
    "\n",
    "print(f\"Before ReLU: {hidden1}\\n\\n\")\n",
    "hidden1 = nn.ReLU()(hidden1)  #  max(0,x), rectified linear unit 线性整流单元, <0表现出整流特征, >0表现出线性特征, 以此得到的稀疏结果, 模拟了生物神经元部分活跃的特性\n",
    "print(f\"After ReLU: {hidden1}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "530afa57-9ac8-4257-9617-fbf72e9cf00e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Sequential 有序的模块化容器, 数据按需通过容器中各模块处理\n",
    "\n",
    "seq_modules = nn.Sequential(\n",
    "    flatten,  # -> (n, 28*28)\n",
    "    layer1,   # -> (n, 20)\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(20, 10)  # -> (n, 10)\n",
    ")\n",
    "input_image = torch.rand(3,28,28)\n",
    "logits = seq_modules(input_image)  # 每个类的数值\n",
    "print(f\"logits: {logits.shape}\")\n",
    "\n",
    "softmax = nn.Softmax(dim=1)   # 每个类的概率\n",
    "pred_probab = softmax(logits)\n",
    "\n",
    "print(f\"Predicted class: {pred_probab.argmax(dim=1)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "edd4c6ca-372d-4153-84cd-71604ccf1fe2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 模型参数查询\n",
    "print(f\"Model structure: {seq_modules}\\n\\n\")\n",
    "\n",
    "for name, param in seq_modules.named_parameters():\n",
    "    print(f\"Layer: {name} | Size: {param.size()} | Values : {param[:2]} \\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d9eae9a-c521-4083-8347-47757d9a0ce9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 自动求导\n",
    "\n",
    "x = torch.ones(5)  # input tensor\n",
    "y = torch.zeros(3)  # expected output\n",
    "w = torch.randn(5, 3, requires_grad=True)\n",
    "b = torch.randn(3, requires_grad=True)\n",
    "z = torch.matmul(x, w)+b\n",
    "loss = torch.nn.functional.binary_cross_entropy_with_logits(z, y)  # 二进制交叉熵, H(y,p)=−[ylogp+(1−y)log(1−p)], 二分类问题, p为特征的概率\n",
    "\n",
    "print(f\"w: {w}\")\n",
    "print(f\"b: {b}\")\n",
    "print(f\"z: {z}\")\n",
    "print(f\"loss: {loss.item()}\")\n",
    "\n",
    "print(f\"Gradient function for z = {z.grad_fn}\")\n",
    "print(f\"Gradient function for loss = {loss.grad_fn}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ab7d43a-d030-4c84-8dc2-14a655c5f570",
   "metadata": {},
   "source": [
    "[信息熵和二进制交叉熵](https://charlesliuyx.github.io/2017/09/11/%E4%BB%80%E4%B9%88%E6%98%AF%E4%BF%A1%E6%81%AF%E7%86%B5%E3%80%81%E4%BA%A4%E5%8F%89%E7%86%B5%E5%92%8C%E7%9B%B8%E5%AF%B9%E7%86%B5/)\n",
    "\n",
    "[什么是信息熵？从存储单位到信息熵](https://www.itheima.com/news/20191105/154251.html)\n",
    "\n",
    "\n",
    "- 宏观态(事件X)下存在多种微观态(可能情况x1, ..., xi), 观察者对属于哪种微观态xi不确定, 信息熵H(xi)就表示这种不确定性;  \n",
    "- 每种微观态出现都有实际概率和观测概率, 接收到的数据, 在(调整观测概率, 排除干扰, 确定情况)后,  \n",
    "  能帮助观察者减少对事件不确定性的才是信息;  \n",
    "- 消除熵 = 获取信息;  \n",
    "- 信息量与事件概率就成反比, 事件概率最大值=1, 此时信息量=0, 符合基本函数中对数特征((1,0)点, 负对数函数即也是反比例函数);  \n",
    "- 当确定宏观态属于某个具体微观态时, 信息熵 = 0;  \n",
    "- 当所有微观态可能概率相同时, 信息熵最大;  \n",
    "\n",
    "[参考逼站解释](https://www.bilibili.com/video/BV15V411W7VB/)\n",
    "- 事件概率 p\n",
    "- 信息量 -log(p)\n",
    "- 信息熵 sum(-p * log(p))\n",
    "- KL散度\n",
    "- 交叉熵\n",
    "- 交叉熵损失函数\n",
    "\n",
    "BCE （Binary Cross Entropy)  [BCEWithLogitsLoss](https://pytorch.org/docs/stable/generated/torch.nn.BCEWithLogitsLoss.html#torch.nn.BCEWithLogitsLoss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b58aba1-3d79-4a6f-b888-d94efee7c6c8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "print(np.log1p([np.e - 1]))  # log1p 计算x+1对底数e的对数, 当前 x = np.e - 1\n",
    "print(np.log([np.e]))  # 底数e\n",
    "print(np.log2([8]))  # 底数2\n",
    "print(np.log10([10000]))  # 底数10\n",
    "print(np.log([8]) / np.log([2]))  #  换底公式与基本对数组合, 计算任意底的对数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fedebf8-7bc7-4ef0-a5dc-29dbf93d4849",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 计算参数的梯度\n",
    "\n",
    "loss.backward()\n",
    "print(w.grad)\n",
    "print(b.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ceb0009-90e6-42b5-bfc7-a97bb07ac7eb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "z = torch.matmul(x, w)+b\n",
    "print(z.requires_grad)\n",
    "\n",
    "with torch.no_grad():  # 禁用梯度跟踪\n",
    "    z = torch.matmul(x, w)+b\n",
    "print(z.requires_grad)\n",
    "\n",
    "z = torch.matmul(x, w)+b\n",
    "z_det = z.detach()   #  分离梯度要求\n",
    "print(z_det.requires_grad)\n",
    "\n",
    "# 如下场景可禁用梯度跟踪:\n",
    "# 1. 标记冷冻参数\n",
    "# 2. 加速计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a43e92f2-08ff-4f0f-ba63-d509ddf6c551",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "inp = torch.eye(4, 5, requires_grad=True)\n",
    "out = ((inp+1)*2).t()\n",
    "print(f\"out\\n{out}\")\n",
    "y = torch.ones_like(out)\n",
    "print(f\"y\\n{y}\")\n",
    "out.backward(y, retain_graph=True)\n",
    "print(f\"First call\\n{inp.grad}\")\n",
    "out.backward(y, retain_graph=True)  # 自动累计前一次的梯度\n",
    "print(f\"\\nSecond call\\n{inp.grad}\")\n",
    "inp.grad.zero_()  # 将当前张量梯度清零, 实际训练中优化器\n",
    "out.backward(y, retain_graph=True)\n",
    "print(f\"\\nCall after zeroing gradients\\n{inp.grad}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py312",
   "language": "python",
   "name": "py312"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
